% Lacey Best-Rowden and Nick Wender
% CSE 802 - Final Project

% SVM on MNIST handwritten digit dataset using 87 features from PCA

images = loadMNISTImages('train-images.idx3-ubyte');
labels = loadMNISTLabels('train-labels.idx1-ubyte');
images = images';   % rows are samples, columns are features

% p is the direction for projection for PCA - each column is one direction
[p,c] = pca(images, 783);
% scatterM = (size(images,1) - 1) * cov(images);
% [eigVects, eigVals] = eigs(scatterM,[],size(scatterM,1)-1);

% 87 dimensions retains at least 90% of the variance (previously found)
imagesPCA = (images - repmat(c, size(images,1), 1)) * p(:,1:87);

% D is a matrix of the images sorted by their class labels 
% (last column of D contains class labels) 
D = sortrows(horzcat(imagesPCA, labels), 88);
d = 87;

% TRAIN the models
trainX = D(:,1:d);
trainY = D(:,d+1);

% defaults: 
%   -s svm_type: 0 -- C-SVC
%   -t kernel_type: 2 -- rbf (exp(-gamma*|u-v|^2)
%   -d degree of kernel fcn: 3
%   -g gamma in kernel fcn: 1/num_features
%   -r coef0 in kernel fcn: 0
%   -c cost: 1
%   -q quiet mode 

defaults = '-s 0 -t 2  -d 3 -g 1/d -r 0 -c 1 -q';
options = '-s 0 -c 1 -t 0 -q';


% GRID SEARCH on (C, gamma) for RBF kernel

%for C = exp(2,-5):exp(2,15)

model_1v1 = svmtrain(trainY, trainX, options);

model_1vR = ovrtrain(trainY, trainX, options);

